#coding=utf-8

'''
Author: Create by hanjt

Date: 2018-12-26

Description: The script is to get the information of business in the website of eleme.

and the url of website is about 'https://www.ele.me'

and the restful api's url is 'https://www.ele.me/restapi/shopping/restaurants?latitude={{31.221631}}&limit={{24}}&longitude={{121.419045}}'

Version of Python: 3.7 (The earliest version of Python must be 3.0. )
'''

import logging
import requests
import json
import pymysql
import time

LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s"
DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
# logging.basicConfig(filename='mkcm.log', level=logging.DEBUG, format=LOG_FORMAT)

'''
返回一个数据库连接
'''
def connect_db():
    return pymysql.connect(host='ip',
                           port=3306,
                           user='root',
                           password='pwd',
                           database='eleme_data_info',
                           charset='utf8')
'''
商家对象模型
'''
class BusinessInfoModel:

    # def __init__(self, rest_id, address, name, flavors, phone, latitude, longitude):
    #     self.__rest_id = rest_id
    #     self.__address = address
    #     self.__name = name
    #     self.__flavors = flavors
    #     self.__phone = phone
    #     self.__latitude = latitude
    #     self.__longitude = longitude
    
    def set_rest_id(self, rest_id):
        self.__rest_id = rest_id
    
    def get_rest_id(self):
        return self.__rest_id

    def set_address(self, address):
        self.__address = address
    
    def get_address(self):
        return self.__address

    def set_name(self, name):
        self.__name = name
    
    def get_name(self):
        return self.__name

    def set_flavors(self, flavors):
        self.__flavors = flavors
    
    def get_flavors(self):
        return self.__flavors

    def set_phone(self, phone):
        self.__phone = phone
    
    def get_phone(self):
        return self.__phone

    def set_latitude(self, latitude):
        self.__latitude = latitude
    
    def get_latitude(self):
        return self.__latitude

    def set_longitude(self, longitude):
        self.__longitude = longitude
    
    def get_longitude(self):
        return self.__longitude


'''
爬虫对象
'''
class MyElmSpider:


    def __init__(self):

        # URL for getting the data from the website of eleme.
        self.url = "https://www.ele.me/restapi/shopping/restaurants"

    '''
    通过GET方式获取远程数据
    '''
    def get_data(self, latitude, longitude, offset):
        headers={
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1",
            "cookie": "ubt_ssid=w8c8bo87pq8gh12y2fsfdk3jnjhasbxd_2018-12-25; _utrace=4b4a9ac49264d16ac34ccd0fcd9aad9b_2018-12-25; cna=GUyfFHz/zE4CAXJffz+686i1; track_id=1545816037|d07c62983a4a35fdc08a6d363fae0eed7061fb6b403873d091|a05ce3d8afdf9a9da837fff193f5e552; USERID=21148384; UTUSER=21148384; SID=3x7UbBecVzkxaDwA2LmpHh7S5AonIGk3JXxA; isg=BCkpARUmUXuwRm3PfASrDIJOONVDXh1sxfhaqcseNJBPkkqkAUX0-jujUDakCrVg; pizza73686f7070696e67=i1hRBLmoJeIPWeq8rUoTjk6LxMQ7SMdqnwd5dhFw-lphdUV8f9uM1OiRJWLovo_K",
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
        }
        try:
            logging.info("==> Offset: " + str(offset))
            result = requests.get(url=self.url, params={"latitude": latitude, 
                                                        "limit": 10000,
                                                        "longitude": longitude, 
                                                        "offset": 0, 
                                                        "terminal": "web"}, headers = headers)
            logging.info(result)
            if 200 != result.status_code:
                logging.error("<== Request Fail")
                return None
            else:
                json_result = json.loads(result.text)
                logging.info("<== Result: " + str(json_result))
                return json_result
        except:
            logging.error("<== Request Error")
            return None    

    '''
    对获取的远程数据进行数据的简单封装
    '''
    def pack_data(self, remote_data):
        data_store = []
        if remote_data is not None:
            for i in range(len(remote_data)):
                r = remote_data[i]
                bm = BusinessInfoModel()
                bm.set_rest_id(r["id"])
                bm.set_address(r["address"])
                bm.set_name(r["name"])
                ps = str(r["phone"]).split(" ")
                phones = ""
                for p in ps:
                    phones = phones + str(p) + ","
                bm.set_phone(phones[:-1])
                bm.set_latitude(r["latitude"])
                bm.set_longitude(r["longitude"])
                flavors = r["flavors"]
                fs = ""
                for j in range(len(flavors)):
                    fs = fs + str(flavors[j]["name"]) + ","
                bm.set_flavors(fs[:-1])
                data_store.append(bm)
        return data_store
    '''
    保存数据至数据库
    '''
    def save_data(self, package_data):
        params = []
        for i in range(len(package_data)):
            t = (
                str(package_data[i].get_rest_id()),
                str(package_data[i].get_address()),
                str(package_data[i].get_name()),
                str(package_data[i].get_phone()),
                str(package_data[i].get_flavors()),
                str(package_data[i].get_latitude()),
                str(package_data[i].get_longitude()),
                int(time.time())
            )
            params.append(t)
        conn = connect_db()
        cur = conn.cursor()
        try:
            cur.executemany("INSERT INTO t_eleme_business_info (rest_id,\
                                                                address,\
                                                                name,\
                                                                phone,\
                                                                flavors,\
                                                                latitude,\
                                                                longitude,\
                                                                create_time)\
                                                   VALUES(%s, %s, %s, %s, %s, %s, %s, %s)", params)
            assert cur.rowcount == len(package_data), 'execute sql error'
            conn.commit()
        except Exception:
            conn.rollback()
            logging.exception('==> Insert into database error')
        finally:
            cur.close()
            conn.close()
    '''
    程序主控制器
    '''
    def main(self):
        logging.info("==> Main")
        for i in range(1000):
            time.sleep(3)
            remote_data = self.get_data("31.221631", "121.419045", i*2)
            package_data = self.pack_data(remote_data)
            self.save_data(package_data)
            logging.info(len(package_data))
        
        

# 程序入口
myElmSpider = MyElmSpider()
myElmSpider.main()



